package co.edu.udistrital.lucene;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Version;

import weka.classifiers.Evaluation;
import weka.classifiers.bayes.NaiveBayes;
import weka.core.Attribute;
import weka.core.Instances;
import weka.core.SerializationHelper;
import weka.core.converters.ArffLoader;

/**
 * This code was originally written for Erik's Lucene intro java.net article
 */
public class Indexer {

	private final List<String> stopWords = Arrays.asList("a's", "able",
			"about", "above", "according", "accordingly", "across", "actually",
			"after", "afterwards", "again", "against", "ain't", "all", "allow",
			"allows", "almost", "alone", "along", "already", "also",
			"although", "always", "am", "among", "amongst", "an", "and",
			"another", "any", "anybody", "anyhow", "anyone", "anything",
			"anyway", "anyways", "anywhere", "apart", "appear", "appreciate",
			"appropriate", "are", "aren't", "around", "as", "aside", "ask",
			"asking", "associated", "at", "available", "away", "awfully", "be",
			"became", "because", "become", "becomes", "becoming", "been",
			"before", "beforehand", "behind", "being", "believe", "below",
			"beside", "besides", "best", "better", "between", "beyond", "both",
			"brief", "but", "by", "c'mon", "c's", "came", "can", "can't",
			"cannot", "cant", "cause", "causes", "certain", "certainly",
			"changes", "clearly", "co", "com", "come", "comes", "concerning",
			"consequently", "consider", "considering", "contain", "containing",
			"contains", "corresponding", "could", "couldn't", "course",
			"currently", "definitely", "described", "despite", "did", "didn't",
			"different", "do", "does", "doesn't", "doing", "don't", "done",
			"down", "downwards", "during", "each", "edu", "eg", "eight", "e.g",
			"either", "else", "elsewhere", "enough", "entirely", "especially",
			"et", "etc", "even", "ever", "every", "everybody", "everyone",
			"everything", "everywhere", "ex", "exactly", "example", "except",
			"far", "few", "fifth", "first", "five", "followed", "following",
			"follows", "for", "former", "formerly", "forth", "four", "from",
			"further", "furthermore", "get", "gets", "getting", "given",
			"gives", "go", "goes", "going", "gone", "got", "gotten",
			"greetings", "had", "hadn't", "happens", "hardly", "has", "hasn't",
			"have", "haven't", "having", "he", "he's", "hello", "help",
			"hence", "her", "here", "here's", "hereafter", "hereby", "herein",
			"hereupon", "hers", "herself", "hi", "him", "himself", "his",
			"hither", "hopefully", "how", "howbeit", "however", "i'd", "i'll",
			"i'm", "i've", "ie", "if", "ignored", "immediate", "in",
			"inasmuch", "inc", "indeed", "indicate", "indicated", "indicates",
			"inner", "insofar", "instead", "into", "inward", "is", "isn't",
			"it", "it'd", "it'll", "it's", "its", "itself", "just", "keep",
			"keeps", "kept", "know", "knows", "known", "last", "lately",
			"later", "latter", "latterly", "least", "less", "lest", "let",
			"let's", "like", "liked", "likely", "little", "look", "looking",
			"looks", "ltd", "mainly", "many", "may", "maybe", "me", "mean",
			"meanwhile", "merely", "might", "more", "moreover", "most",
			"mostly", "much", "must", "my", "myself", "name", "namely", "nd",
			"near", "nearly", "necessary", "need", "needs", "neither", "never",
			"nevertheless", "new", "next", "nine", "no", "nobody", "non",
			"none", "noone", "nor", "normally", "not", "nothing", "novel",
			"now", "nowhere", "obviously", "of", "off", "often", "oh", "ok",
			"okay", "old", "on", "once", "one", "ones", "only", "onto", "or",
			"other", "others", "otherwise", "ought", "our", "ours",
			"ourselves", "out", "outside", "over", "overall", "own",
			"particular", "particularly", "per", "perhaps", "placed", "please",
			"plus", "possible", "presumably", "probably", "provides", "que",
			"quite", "qv", "rather", "rd", "re", "really", "reasonably",
			"regarding", "regardless", "regards", "relatively", "respectively",
			"right", "said", "same", "saw", "say", "saying", "says", "second",
			"secondly", "see", "seeing", "seem", "seemed", "seeming", "seems",
			"seen", "self", "selves", "sensible", "sent", "serious",
			"seriously", "seven", "several", "shall", "she", "should",
			"shouldn't", "since", "six", "so", "some", "somebody", "somehow",
			"someone", "something", "sometime", "sometimes", "somewhat",
			"somewhere", "soon", "sorry", "specified", "specify", "specifying",
			"still", "sub", "such", "sup", "sure", "t's", "take", "taken",
			"tell", "tends", "th", "than", "thank", "thanks", "thanx", "that",
			"that's", "thats", "the", "their", "theirs", "them", "themselves",
			"then", "thence", "there", "there's", "thereafter", "thereby",
			"therefore", "therein", "theres", "thereupon", "these", "they",
			"they'd", "they'll", "they're", "they've", "think", "third",
			"this", "thorough", "thoroughly", "those", "though", "three",
			"through", "throughout", "thru", "thus", "to", "together", "too",
			"took", "toward", "towards", "tried", "tries", "truly", "try",
			"trying", "twice", "two", "un", "under", "unfortunately", "unless",
			"unlikely", "until", "unto", "up", "upon", "us", "use", "used",
			"useful", "uses", "using", "usually", "value", "various", "very",
			"via", "viz", "vs", "want", "wants", "was", "wasn't", "way", "we",
			"we'd", "we'll", "we're", "we've", "welcome", "well", "went",
			"were", "weren't", "what", "what's", "whatever", "when", "whence",
			"whenever", "where", "where's", "whereafter", "whereas", "whereby",
			"wherein", "whereupon", "wherever", "whether", "which", "while",
			"whither", "who", "who's", "whoever", "whole", "whom", "whose",
			"why", "will", "willing", "wish", "with", "within", "without",
			"won't", "wonder", "would", "would", "wouldn't", "yes", "yet",
			"you", "you'd", "you'll", "you're", "you've", "your", "yours",
			"yourself", "yourselves", "zero", "a", "b", "c", "d", "e", "f",
			"g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s",
			"t", "u", "v", "w", "x", "y", "z", "1", "2", "3", "4", "5", "6",
			"7", "8", "9", "0", "titulo", "abstract", "terminos", "categoria",
			"paper");

	private IndexWriter writer;

	private Directory dir;

	/**
	 * Contains the stopwords used with the StopFilter.
	 */
	private Set<Object> stopTable = new HashSet<Object>();

	
	public static void main(String[] args) throws Exception {
		String indexDir = "D:\\LuceneAcmDocsIndexDir"; // 1
		String dataDir = "D:\\LuceneAcmDocsDir"; // 2
		long start = System.currentTimeMillis();
		Indexer indexer = new Indexer(indexDir);
		int numIndexed = indexer.index(new File(dataDir));
		indexer.close();
		String archivo = indexer.convertToArff();
		indexer.clasifyWeka(archivo);
		long end = System.currentTimeMillis();
		System.out.println("Indexing " + numIndexed + " files took "
				+ (end - start) + " milliseconds");
	}

	public Indexer(String indexDirName) throws IOException {
		File indexDir = new File(indexDirName);
		if (indexDir.exists()) {
			deleteDir(indexDir);
		}
		dir = new SimpleFSDirectory(indexDir, null);
		stopTable = StopFilter.makeStopSet(Version.LUCENE_36, stopWords);
		EnglishAnalyzer analyzer = new EnglishAnalyzer(Version.LUCENE_36,
				stopTable);
		IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36,
				analyzer);
		writer = new IndexWriter(dir, config);
		// StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_29);
		// writer = new IndexWriter(dir, analyzer, MaxFieldLength.UNLIMITED);
	}

	public void deleteDir(File firstFile) throws IOException {
		if (firstFile.isDirectory()) {
			File[] files = firstFile.listFiles();

			for (int i = 0; i < files.length; i++) {
				File f = files[i];
				deleteDir(f);
			}

		}
		firstFile.delete();
	}

	public void close() throws IOException {
		writer.close(); // 4
	}

	public int index(File firstFile) throws Exception {
		File[] files = firstFile.listFiles();

		for (int i = 0; i < files.length; i++) {
			File f = files[i];
			if (f.isDirectory()) {
				index(f);
			} else if (!f.isHidden() && f.exists() && f.canRead()) {
				// && acceptFile(f)) {
				indexFile(f);
			}
		}
		return writer.numDocs(); // 5
	}

	protected boolean acceptFile(File f) { // 6
		return f.getName().endsWith(".txt");
	}

	protected Document getDocument(File f) throws Exception {
		Document doc = new Document();
		FileInputStream fstream = new FileInputStream(f);
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(fstream);
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		// Read File Line By Line
		String categoria = "";
		while ((strLine = br.readLine()) != null) {
			int indice = strLine.lastIndexOf("4-Categoria");
			if (indice > -1) {
				categoria = (strLine.substring(indice + 11)).trim();
				if ((categoria == null || categoria.isEmpty())
						&& (strLine = br.readLine()) != null) {
					categoria = strLine;
				}
			}
		}
		if (categoria == null || categoria.isEmpty()) {
			categoria = "General";
		}
		// Close the input stream
		br.close();
		in.close();
		fstream.close();
		doc.add(new Field("contents", new FileReader(f), TermVector.YES)); // 7
		doc.add(new Field("filename", f.getCanonicalPath(), // 8
				Field.Store.YES, Field.Index.NOT_ANALYZED));
		doc.add(new Field("categoria", categoria, Field.Store.YES,
				Field.Index.NOT_ANALYZED));
		return doc;
	}

	private void indexFile(File f) throws Exception {
		System.out.println("Indexing " + f.getCanonicalPath());
		Document doc = getDocument(f);
		if (doc != null) {
			writer.addDocument(doc); // 9
		}
	}

	public String convertToArff() throws Exception {
		LuceneToArffConverter converter = new LuceneToArffConverter();
		return converter.convert(dir);
	}

	public void clasifyWeka(String dir) {
		ArffLoader loader = new ArffLoader();
		File archivo = new File(dir);
		try {
			loader.setSource(archivo);
			Instances dataSource = loader.getDataSet();
			int classIndex = dataSource.classIndex();
			System.out.println("classIndex: " + classIndex);
			int numAttributes = dataSource.numAttributes();
			Attribute last = dataSource.attribute(numAttributes - 1);
			System.out.println("Last: " + last.name());
			System.out.println("numAttributes: " + numAttributes);
			dataSource.setClassIndex(numAttributes - 1);
			String[] options = new String[2];
			options[0] = "-C 0.25"; // unpruned tree
			options[1] = "-M 2";
			NaiveBayes naybe = new NaiveBayes();
			Evaluation eval = new Evaluation(dataSource);
			naybe.buildClassifier(dataSource);
			eval.evaluateModel(naybe, dataSource);
			// eval.crossValidateModel(naybe, dataSource, 10, new Random(1));
			System.out.println(eval.toSummaryString("\nResults\n\n", false));
			// serialize model
			 SerializationHelper.write("D:\\naybe2.model", naybe);
			 NaiveBayes naybe2 = (NaiveBayes) SerializationHelper.read("D:\\naybe2.model");
			Instances unlabeled = new Instances(
                    new BufferedReader(
                      new FileReader("D:\\Nuevos.arff")));
			unlabeled.setClassIndex(unlabeled.numAttributes() - 1);
			Instances labeled = new Instances(unlabeled);
			for (int i = 0; i < unlabeled.numInstances() ; i++) {
				   double clsLabel = naybe2.classifyInstance(unlabeled.instance(i));
				   labeled.instance(i).setClassValue(clsLabel);
			}
			 // save labeled data
			 BufferedWriter writer = new BufferedWriter(
			                           new FileWriter("D:\\NuevosClasificados.arff"));
			 writer.write(labeled.toString());
			 writer.newLine();
			 writer.flush();
			 writer.close();			/*
			J48 tree = new J48(); // new instance of tree
			tree.setOptions(options); // set the options
			tree.buildClassifier(dataSource); // build classifier IBk ibk =
			new IBk(); String[] options2 = {"-K 1", "-W 0",
			"-A \"weka.core.neighboursearch.LinearNNSearch -A \"weka.core.EuclideanDistance"
			, "-R first-last\""}; ibk.setOptions(options2);
			ibk.buildClassifier(dataSource);
			 */
		} catch (IOException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}